\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{enumerate}
\usepackage[left=2cm,right=2cm,top=2cm,bottom=2cm]{geometry}
\title{Solutions for Homework 1}
\date{}

\def\MN{{\mathcal N}}

\begin{document}
\maketitle
\begin{enumerate}
\item
If $\lim \limits_{n\rightarrow\infty}a_n=a$, show that $\lim \limits_{n\rightarrow\infty}\left(1+\frac{a_n}{n}\right)^n=e^{a}$.
\begin{proof}
$$\lim \limits_{n\rightarrow\infty}\left(1+\frac{a_n}{n}\right)^n  = \lim \limits_{n\rightarrow\infty}\left(\left(1+\frac{1}{n/a_n}\right)^{n/a_n}\right)^{a_n} = e^a$$
\end{proof}

\item
Prove the Stirling Formula.
\begin{eqnarray*}
\lim_{p\rightarrow\infty}\frac{\ln\Gamma(p)}{\frac{1}{2}\ln(2\pi)+(p-\frac{1}{2})\ln{p}-p}&=&1\\
\lim_{p\rightarrow\infty}\frac{\Gamma(p)}{(2\pi)^{\frac{1}{2}}p^{p-\frac{1}{2}}e^{-p}}&=&1
\end{eqnarray*}
\begin{proof}
The Gamma function is :
\begin{eqnarray*}
\Gamma(p+1)  = &\int_0^{\infty}x^pe^{-x}dx \\
= &\int_0^{\infty}e^{p\ln x-x}dx \\
= &\int_0^{\infty}e^{p(\ln x-\frac{x}{p})}dx \\
\end{eqnarray*}
Let $f(x) = \ln x-\frac{x}{p}$. It's twice differentiable function on $[0,\infty)$.
$f'(x) = \frac{1}{x}-\frac{1}{p}$, so $x=p$ is the unique maximum point and $f''(x) = -\frac{1}{x^2}<0$. Then, according to Laplace's method \footnote{http://en.wikipedia.org/wiki/Laplace\%27s\_method}, we have:
$$\lim_{p \to +\infty}  \frac{\int_0^\infty e^{pf(x)} \, dx}{\left( e^{pf(p)}\sqrt{\frac{2 \pi}{p (-f''(p))}} \right)} =1 $$
namely,
\begin{eqnarray*}
& \lim\limits_{ p \to +\infty}  \frac{\Gamma(p)}{\left( e^{p(\ln p-\frac{p}{p})}\sqrt{\frac{2 \pi}{p (\frac{1}{p^2})}} \right)}  &= 1 \\
\Longrightarrow & \lim\limits_{p \to +\infty}  \frac{\Gamma(p)}{(2\pi)^{\frac{1}{2}}p^{p-\frac{1}{2}}e^{-p}} &= 1 \\
\end{eqnarray*}
It is easy to prove the other formula by L'Hospital's rule.
\end{proof}

\item
$\sum\limits_{k=0}^\infty \mbox{Gamma}(x | k+\rho+1, \beta) \mbox{Poisson} (k | \lambda),  \rho\mbox{ is a constant},\rho>-1$
\begin{eqnarray*}
&& \sum\limits_{k=0}^\infty \mbox{Gamma}(x | k+\rho+1, \beta) \mbox{Poisson} (k | \lambda) \\
& = & \sum\limits_{k=0}^\infty \frac{\beta^{k+\rho+1}}{\Gamma(k+\rho+1)}x^{k+\rho}e^{-\beta x}e^{-\lambda}\frac{\lambda^{k}}{k!} \\
& = & \frac{\beta e^{-\lambda-\beta x}}{\lambda^{\rho}}\sum\limits_{k=0}^\infty \frac{1}{\Gamma(k+\rho+1)k!}(\lambda\beta x)^{k+\rho} \\
& = & \beta e^{-\lambda-\beta x}\sqrt{\left(\beta x/\lambda\right)^\rho}\sum\limits_{k=0}^\infty \frac{1}{\Gamma(k+\rho+1)k!}\left(\sqrt{\lambda\beta x}\right)^{2k+\rho} \\
& = & \beta e^{-\lambda-\beta x}\sqrt{\left(\beta x/\lambda\right)^\rho} I_\rho\left(2\sqrt{\lambda\beta x}\right) \\
\end{eqnarray*}
where $I_\rho$ is the modified Bessel functions of the first kind, which is defined by
$$I_\rho(x) =\sum_{k=0}^\infty \frac{1}{k! \Gamma(k+\rho+1)}\left(\frac{x}{2}\right)^{2k+\rho}$$

\item Compute the following integrals:
\begin{enumerate}
\item $u_0 = \int_{-\infty}^{\infty} \Phi(x) \MN(x|\mu, \sigma^2) dx$
\item $u_1 = \int_{-\infty}^{\infty} \Phi(x) \MN(x|\mu, \sigma^2)x dx$
\item $u_2 = \int_{-\infty}^{\infty} \Phi(x) \MN(x|\mu, \sigma^2)(x-m_1)^2 dx$

where $\Phi(x) = \int_{-\infty}^{x} \frac{1}{\sqrt{2\pi}} e^{-\frac{t^2}{2}} dt$
\end{enumerate}
\begin{enumerate}
\item 
\begin{eqnarray*}
u_0 &=& \int_{-\infty}^{\infty} \left(\int_{-\infty}^{x} \frac{1}{\sqrt{2\pi}} e^{-\frac{t^2}{2}}\right) \frac{1}{\sqrt{2\pi}\sigma} e^{-\frac{(x-\mu)^2}{2\sigma^2}} dx \\
&=& \int_{-\infty}^{\infty} \left(\int_{-\infty}^{\sigma y+\mu} \frac{1}{\sqrt{2\pi}} e^{-\frac{t^2}{2}}\right) \frac{1}{\sqrt{2\pi}} e^{-\frac{y^2}{2}} dy   \\
&& \mbox{(use }y=(x-\mu)/\sigma-\mbox{ to replace x, then }Y\sim \MN(0,1))\\
&=&\int_{-\infty}^{\infty} \Phi(\sigma y+\mu) \MN(y|0, 1) dy \\
&=&\int_{-\infty}^{\infty} P(K\le \sigma Y+\mu | Y=y) \varphi(y) dy  \\
&&(K \sim \MN(0,1) \mbox{ and is independent of }Y, \varphi(x)\mbox{ is the pdf of }\MN(0,1)) \\
&=&P(K\le \sigma Y+\mu) \\
&&\mbox{(use the law of total probability)}\\
&=&P(K-\sigma Y\le \mu) \\
\end{eqnarray*}
Let $Z=K-\sigma Y$, then $Z\sim \MN(0,1+\sigma^2)$, so $\mu_0=P(K-\sigma Y\le \mu) = P(Z\le \mu) = \Phi(\frac{\mu}{1+\sigma^2})$.
\item 
\begin{eqnarray*}
&& \frac{\partial u_0}{\partial \mu} = \frac{\partial \Phi(\frac{\mu}{\sqrt{1+\sigma^2}})}{\partial \mu} \\
&\Longrightarrow &\int_{-\infty}^{\infty}\Phi(x)\MN(\mu,\sigma^2)\frac{x-\mu}{\sigma^2}dx = \varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\frac{1}{\sqrt{1+\sigma^2}} \\
&\Longrightarrow &\int_{-\infty}^{\infty}\Phi(x)\MN(\mu,\sigma^2)(x-\mu)dx = \varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\frac{\sigma^2}{\sqrt{1+\sigma^2}} \\
&\Longrightarrow &u_1 = \mu u_0+ \varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\frac{\sigma^2}{\sqrt{1+\sigma^2}} \\
\end{eqnarray*}
\item
\begin{eqnarray*}
&& u_2 = \int_{-\infty}^{\infty} \Phi(x) \MN(x|\mu, \sigma^2)x^2 - 2m_1 u_1 + m_1^2u_0 \\
\end{eqnarray*}
and 
\begin{eqnarray*}
&& \frac{\partial u_1}{\partial \mu} = \frac{\partial(\mu u_0+ \varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\frac{\sigma^2}{\sqrt{1+\sigma^2}})}{\partial \mu} \\
&\Longrightarrow &\int_{-\infty}^{\infty}\Phi(x)\MN(\mu,\sigma^2)x\frac{x-\mu}{\sigma^2}dx = -\frac{\mu\sigma^2}{\sqrt{(1+\sigma^2)^{3}}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})+u_0+\frac{\mu}{\sqrt{1+\sigma^2}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}}) \\
&\Longrightarrow &\int_{-\infty}^{\infty}\Phi(x)\MN(\mu,\sigma^2)x(x-\mu)dx = \sigma^2\left(-\frac{\mu\sigma^2}{\sqrt{(1+\sigma^2)^{3}}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})+u_0+\frac{\mu}{\sqrt{1+\sigma^2}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\right) \\
&\Longrightarrow &\int_{-\infty}^{\infty} \Phi(x) \MN(x|\mu, \sigma^2)x^2 = \mu u_1+ \sigma^2\left(-\frac{\mu\sigma^2}{\sqrt{(1+\sigma^2)^{3}}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})+u_0+\frac{\mu}{\sqrt{1+\sigma^2}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\right) \\
\end{eqnarray*}
Hence,
\begin{eqnarray*}
u_2 &=& \mu u_1+ \sigma^2\left(-\frac{\mu\sigma^2}{\sqrt{(1+\sigma^2)^{3}}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})+u_0+\frac{\mu}{\sqrt{1+\sigma^2}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})\right)-2m_1u_1+m_1^2u_0 \\
&=& (m_1^2-\mu^2)u_0 +2(\mu-m_1)u_1+\sigma^2\left(-\frac{\mu\sigma^2}{\sqrt{(1+\sigma^2)^{3}}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}})+u_0\right) \\
&=& (m_1^2-\mu^2+\sigma^2)u_0 +2(\mu-m_1)u_1-\frac{\mu\sigma^4}{\sqrt{(1+\sigma^2)^{3}}}\varphi(\frac{\mu}{\sqrt{1+\sigma^2}}) \\
\end{eqnarray*}
\end{enumerate}
\item $f(x; \theta) = \theta^x(1-\theta)^{1-x}$, $0 < \theta < 1$.
\begin{enumerate}
\item Compute Jeffreys prior about $\theta$.
\item If $\theta = \sin^2 \alpha$, compute Jeffreys prior about $\alpha$.
\end{enumerate}
\begin{enumerate}
\item 
\begin{align*}p(\theta) & \propto \sqrt{I(\theta)} = \sqrt{\operatorname{E}\!\left[ \left( \frac{d}{d\theta} \log f(x|\theta) \right)^2\right]} = \sqrt{\operatorname{E}\!\left[ \left( \frac{x}{\theta} - \frac{1-x}{1-\theta}\right)^2 \right]} \\ & = \sqrt{\theta \left( \frac{1}{\theta} - \frac{0}{1-\theta}\right)^2 + (1-\theta)\left( \frac{0}{\theta} - \frac{1}{1-\theta}\right)^2} = \frac{1}{\sqrt{\theta(1-\theta)}}\,.\end{align*} 
\item 
\begin{align*}
p(\alpha) \propto&  p(\theta)|\frac{d\theta}{d\alpha}| \\
\propto & \frac{1}{|\sin\alpha\cos\alpha|}|2\sin \alpha\cos \alpha| \\
\propto & 1 \\
\end{align*}
\end{enumerate}
\end{enumerate}
\end{document}